<!DOCTYPE html>



  


<html class="theme-next pisces use-motion" lang="zh-Hans">
<head>
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />







<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.3" rel="stylesheet" type="text/css" />


  <link rel="apple-touch-icon" sizes="180x180" href="/images/avatar.jpg?v=5.1.3">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/avatar.jpg?v=5.1.3">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/avatar.jpg?v=5.1.3">


  <link rel="mask-icon" href="/images/avatar.jpg?v=5.1.3" color="#222">





  <meta name="keywords" content="Kubernetes,Docker," />










<meta name="description" content="环境 7台虚拟机，三master三node，一台作为Docker私库 VIP 192.168.11.222   master 192.168.11.31 192.168.11.32 192.168.11.33   node 192.168.11.34   192.168.11.35 192.168.11.36   Harbor registry 192.168.11.200">
<meta name="keywords" content="Kubernetes,Docker">
<meta property="og:type" content="article">
<meta property="og:title" content="Kubernetes 记录">
<meta property="og:url" content="http://fighterhit.github.io/2018/12/04/other_notes/Kubernetes-记录/index.html">
<meta property="og:site_name" content="Fighter&#39;s Blog">
<meta property="og:description" content="环境 7台虚拟机，三master三node，一台作为Docker私库 VIP 192.168.11.222   master 192.168.11.31 192.168.11.32 192.168.11.33   node 192.168.11.34   192.168.11.35 192.168.11.36   Harbor registry 192.168.11.200">
<meta property="og:locale" content="zh-Hans">
<meta property="og:updated_time" content="2018-12-04T06:37:05.954Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Kubernetes 记录">
<meta name="twitter:description" content="环境 7台虚拟机，三master三node，一台作为Docker私库 VIP 192.168.11.222   master 192.168.11.31 192.168.11.32 192.168.11.33   node 192.168.11.34   192.168.11.35 192.168.11.36   Harbor registry 192.168.11.200">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Pisces',
    version: '5.1.3',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://fighterhit.github.io/2018/12/04/other_notes/Kubernetes-记录/"/>





  <title>Kubernetes 记录 | Fighter's Blog</title><!-- hexo-inject:begin --><!-- hexo-inject:end -->
  








</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <!-- hexo-inject:begin --><!-- hexo-inject:end --><div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">Fighter's Blog</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle">深度沉迷学习</p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://fighterhit.github.io/2018/12/04/other_notes/Kubernetes-记录/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Fighter.">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Fighter's Blog">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">Kubernetes 记录</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2018-12-04T11:17:30+08:00">
                2018-12-04
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/其它笔记/" itemprop="url" rel="index">
                    <span itemprop="name">其它笔记</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          

          
            <span class="post-meta-divider">|</span>
            <span class="page-pv"><i class="fa fa-file-o"></i>
            <span class="busuanzi-value" id="busuanzi_value_page_pv" ></span>
            </span>
          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <h2 id="环境"><a href="#环境" class="headerlink" title="环境"></a>环境</h2><ul>
<li>7台虚拟机，三master三node，一台作为Docker私库</li>
<li>VIP<ul>
<li>192.168.11.222</li>
</ul>
</li>
<li>master<ul>
<li>192.168.11.31</li>
<li>192.168.11.32</li>
<li>192.168.11.33</li>
</ul>
</li>
<li>node<ul>
<li>192.168.11.34  </li>
<li>192.168.11.35</li>
<li>192.168.11.36</li>
</ul>
</li>
<li>Harbor registry<ul>
<li>192.168.11.200</li>
</ul>
</li>
</ul>
<a id="more"></a>
<h3 id="系统配置"><a href="#系统配置" class="headerlink" title="系统配置"></a>系统配置</h3><ul>
<li>启动SSH并开机自启动</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">service sshd start</div><div class="line">systemctl enable sshd.service</div></pre></td></tr></table></figure>
<ul>
<li>关闭防火墙和selinux</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">systemctl stop firewalld  &amp;&amp; systemctl disable firewalld</div><div class="line">vim /etc/selinux/config</div><div class="line">修改SELINUX=disabled</div><div class="line">setenforce 0</div><div class="line">getenforce</div></pre></td></tr></table></figure>
<ul>
<li>关闭swap</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">swapoff -a &amp;&amp; sed -i &apos;/swap/d&apos; /etc/fstab</div></pre></td></tr></table></figure>
<ul>
<li>配置系统路由参数,防止kubeadm报路由警告</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">echo &quot;net.bridge.bridge-nf-call-ip6tables = 1&quot; &gt;&gt; /etc/sysctl.conf</div><div class="line">echo &quot;net.bridge.bridge-nf-call-iptables = 1&quot; &gt;&gt; /etc/sysctl.conf</div><div class="line">cat  /etc/sysctl.conf 确认</div><div class="line">sysctl -p</div></pre></td></tr></table></figure>
<ul>
<li>修改节点名</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div></pre></td><td class="code"><pre><div class="line">hostnamectl set-hostname k8s1</div><div class="line">hostnamectl set-hostname k8s2</div><div class="line">hostnamectl set-hostname k8s3</div><div class="line">hostnamectl set-hostname k8s4</div><div class="line">hostnamectl set-hostname k8s5</div><div class="line">hostnamectl set-hostname k8s6</div><div class="line">hostnamectl set-hostname k8s-deploy</div></pre></td></tr></table></figure>
<ul>
<li>修改/etc/hosts文件</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div></pre></td><td class="code"><pre><div class="line">192.168.11.31   k8s1</div><div class="line">192.168.11.32   k8s2</div><div class="line">192.168.11.33   k8s3</div><div class="line">192.168.11.34   k8s4</div><div class="line">192.168.11.35   k8s5</div><div class="line">192.168.11.36   k8s6</div><div class="line">192.168.11.200  k8s-deploy</div><div class="line"></div><div class="line">192.30.253.113 github.com</div><div class="line">192.30.252.131 github.com</div><div class="line">185.31.16.185 github.global.ssl.fastly.net</div><div class="line">74.125.237.1 dl-ssl.google.com</div><div class="line">173.194.127.200 groups.google.com</div><div class="line">192.30.252.131 github.com</div><div class="line">185.31.16.185 github.global.ssl.fastly.net</div><div class="line">74.125.128.95 ajax.googleapis.com</div></pre></td></tr></table></figure>
<ul>
<li>配置免密钥登录</li>
</ul>
<p>每个点都执行</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line">ssh-keygen</div><div class="line">ssh-copy-id k8s1</div><div class="line">ssh-copy-id k8s2</div><div class="line">ssh-copy-id k8s3</div><div class="line">ssh-copy-id k8s4</div><div class="line">ssh-copy-id k8s5</div><div class="line">ssh-copy-id k8s6</div><div class="line">ssh-copy-id k8s-deploy</div></pre></td></tr></table></figure>
<h2 id="安装-keepalived（只对master节点操作）"><a href="#安装-keepalived（只对master节点操作）" class="headerlink" title="安装 keepalived（只对master节点操作）"></a>安装 keepalived（只对master节点操作）</h2><h3 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">yum install -y keepalived</div></pre></td></tr></table></figure>
<h3 id="修改配置文件-etc-keepalived-keepalived-conf"><a href="#修改配置文件-etc-keepalived-keepalived-conf" class="headerlink" title="修改配置文件 /etc/keepalived/keepalived.conf"></a>修改配置文件 /etc/keepalived/keepalived.conf</h3><p>备份后每个节点修改相应内容</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div></pre></td><td class="code"><pre><div class="line">global_defs &#123;</div><div class="line">   router_id LVS_k8s</div><div class="line">&#125;</div><div class="line">vrrp_script CheckK8sMaster &#123;</div><div class="line">    script &quot;curl -k https://192.168.11.222:6443&quot;		//VIP的IP</div><div class="line">    interval 3</div><div class="line">    timeout 9</div><div class="line">    fall 2</div><div class="line">    rise 2</div><div class="line">&#125;</div><div class="line">vrrp_instance VI_1 &#123;</div><div class="line">    state MASTER</div><div class="line">    interface eno16777736					//ifconfig上面的对应IP项，见下图</div><div class="line">    virtual_router_id 61</div><div class="line">    priority 120</div><div class="line">    advert_int 1</div><div class="line">    mcast_src_ip 192.168.11.31				//本机的IP</div><div class="line">    nopreempt</div><div class="line">    authentication &#123;</div><div class="line">        auth_type PASS</div><div class="line">        auth_pass sqP05dQgMSlzrxHj</div><div class="line">    &#125;</div><div class="line">    unicast_peer &#123;</div><div class="line">        #192.168.11.31						//master集群的，将本机的注释掉</div><div class="line">        192.168.11.32</div><div class="line">        192.168.11.33</div><div class="line">    &#125;</div><div class="line">    virtual_ipaddress &#123;</div><div class="line">        192.168.11.222/24						//VIP的IP</div><div class="line">    &#125;</div><div class="line">    track_script &#123;</div><div class="line">        CheckK8sMaster</div><div class="line">    &#125;</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<h3 id="依次启动"><a href="#依次启动" class="headerlink" title="依次启动"></a>依次启动</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">systemctl enable keepalived &amp;&amp; systemctl restart keepalived</div><div class="line">systemctl status keepalived</div></pre></td></tr></table></figure>
<p>修改相应的IP配置，启动起来，应该能看到两个节点是BACKUP STATE</p>
<blockquote>
<p>建立install目录，将准备的文件都拷贝到这个路径下，接下来安装都以install目录内文件为准</p>
</blockquote>
<h2 id="安装Etcd-https（只对master节点操作）"><a href="#安装Etcd-https（只对master节点操作）" class="headerlink" title="安装Etcd https（只对master节点操作）"></a>安装Etcd https（只对master节点操作）</h2><ul>
<li>k8s1 执行</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">export NODE_NAME=k8s1</div><div class="line">export NODE_IP=192.168.11.31</div><div class="line">export NODE_IPS=&quot;192.168.11.31 192.168.11.32 192.168.11.33&quot;</div><div class="line">export ETCD_NODES=k8s1=https://192.168.11.31:2380,k8s2=https://192.168.11.32:2380,k8s3=https://192.168.11.33:2380</div></pre></td></tr></table></figure>
<ul>
<li>k8s2 执行</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">export NODE_NAME=k8s2</div><div class="line">export NODE_IP=192.168.11.32</div><div class="line">export NODE_IPS=&quot;192.168.11.31 192.168.11.32 192.168.11.33&quot;</div><div class="line">export ETCD_NODES=k8s1=https://192.168.11.31:2380,k8s2=https://192.168.11.32:2380,k8s3=https://192.168.11.33:2380</div></pre></td></tr></table></figure>
<ul>
<li>k8s3 执行</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">export NODE_NAME=k8s3</div><div class="line">export NODE_IP=192.168.11.33</div><div class="line">export NODE_IPS=&quot;192.168.11.31 192.168.11.32 192.168.11.33&quot;</div><div class="line">export ETCD_NODES=k8s1=https://192.168.11.31:2380,k8s2=https://192.168.11.32:2380,k8s3=https://192.168.11.33:2380</div></pre></td></tr></table></figure>
<h3 id="Etcd-证书"><a href="#Etcd-证书" class="headerlink" title="Etcd 证书"></a>Etcd 证书</h3><ul>
<li>创建 CA 证书和秘钥</li>
</ul>
<p>安装cfssl， CloudFlare 的 PKI 工具集 cfssl 来生成 Certificate Authority (CA) 证书和秘钥文件<br>如果不希望将cfssl工具安装到部署主机上，可以在其他的主机上进行该步骤，生成以后将证书拷贝到部署etcd的主机上即可。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line">chmod +x cfssl_linux-amd64</div><div class="line">mv cfssl_linux-amd64 /usr/local/bin/cfssl</div><div class="line"></div><div class="line">chmod +x cfssljson_linux-amd64</div><div class="line">mv cfssljson_linux-amd64 /usr/local/bin/cfssljson</div><div class="line"></div><div class="line">chmod +x cfssl-certinfo_linux-amd64</div><div class="line">mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo</div></pre></td></tr></table></figure>
<ul>
<li>生成 ETCD 的 TLS 秘钥和证书</li>
</ul>
<blockquote>
<p>ca-config.json：可以定义多个 profiles，分别指定不同的过期时间、使用场景等参数；后续在签名证书时使用某个 profile；<br>signing：表示该证书可用于签名其它证书；生成的 ca.pem 证书中 CA=TRUE；<br>server auth：表示 client 可以用该 CA 对 server 提供的证书进行验证；<br>client auth：表示 server 可以用该 CA 对 client 提供的证书进行验证；</p>
</blockquote>
<p>为了保证通信安全，客户端(如 etcdctl) 与 etcd 集群、etcd 集群之间的通信需要使用 TLS 加密，本节创建 etcd TLS 加密所需的证书和私钥。<br>创建 CA 配置文件：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div></pre></td><td class="code"><pre><div class="line">cat &gt;  ca-config.json &lt;&lt;EOF</div><div class="line">&#123;</div><div class="line">&quot;signing&quot;: &#123;</div><div class="line">&quot;default&quot;: &#123;</div><div class="line">  &quot;expiry&quot;: &quot;8760h&quot;</div><div class="line">&#125;,</div><div class="line">&quot;profiles&quot;: &#123;</div><div class="line">  &quot;kubernetes&quot;: &#123;</div><div class="line">    &quot;usages&quot;: [</div><div class="line">        &quot;signing&quot;,</div><div class="line">        &quot;key encipherment&quot;,</div><div class="line">        &quot;server auth&quot;,</div><div class="line">        &quot;client auth&quot;</div><div class="line">    ],</div><div class="line">    &quot;expiry&quot;: &quot;8760h&quot;</div><div class="line">  &#125;</div><div class="line">&#125;</div><div class="line">&#125;</div><div class="line">&#125;</div><div class="line">EOF</div></pre></td></tr></table></figure>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div></pre></td><td class="code"><pre><div class="line">cat &gt;  ca-csr.json &lt;&lt;EOF</div><div class="line">&#123;</div><div class="line">&quot;CN&quot;: &quot;kubernetes&quot;,</div><div class="line">&quot;key&quot;: &#123;</div><div class="line">&quot;algo&quot;: &quot;rsa&quot;,</div><div class="line">&quot;size&quot;: 2048</div><div class="line">&#125;,</div><div class="line">&quot;names&quot;: [</div><div class="line">&#123;</div><div class="line">  &quot;C&quot;: &quot;CN&quot;,</div><div class="line">  &quot;ST&quot;: &quot;iie&quot;,</div><div class="line">  &quot;L&quot;: &quot;iie&quot;,</div><div class="line">  &quot;O&quot;: &quot;k8s&quot;,</div><div class="line">  &quot;OU&quot;: &quot;System&quot;</div><div class="line">&#125;</div><div class="line">]</div><div class="line">&#125;</div><div class="line">EOF</div></pre></td></tr></table></figure>
<blockquote>
<p>“CN”：Common Name，kube-apiserver 从证书中提取该字段作为请求的用户名 (User Name)；浏览器使用该字段验证网站是否合法；<br>“O”：Organization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)；</p>
</blockquote>
<ul>
<li>生成 CA 证书和私钥：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cfssl gencert -initca ca-csr.json | cfssljson -bare ca</div><div class="line">ls ca*</div></pre></td></tr></table></figure>
<ul>
<li>创建 etcd 证书签名请求：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div></pre></td><td class="code"><pre><div class="line">cat &gt; etcd-csr.json &lt;&lt;EOF</div><div class="line">&#123;</div><div class="line">  &quot;CN&quot;: &quot;etcd&quot;,</div><div class="line">  &quot;hosts&quot;: [</div><div class="line">    &quot;127.0.0.1&quot;,</div><div class="line">    &quot;192.168.11.31&quot;,</div><div class="line">    &quot;192.168.11.32&quot;,</div><div class="line">    &quot;192.168.11.33&quot;</div><div class="line">  ],</div><div class="line">  &quot;key&quot;: &#123;</div><div class="line">    &quot;algo&quot;: &quot;rsa&quot;,</div><div class="line">    &quot;size&quot;: 2048</div><div class="line">  &#125;,</div><div class="line">  &quot;names&quot;: [</div><div class="line">    &#123;</div><div class="line">      &quot;C&quot;: &quot;CN&quot;,</div><div class="line">      &quot;ST&quot;: &quot;iie&quot;,</div><div class="line">      &quot;L&quot;: &quot;iie&quot;,</div><div class="line">      &quot;O&quot;: &quot;k8s&quot;,</div><div class="line">      &quot;OU&quot;: &quot;System&quot;</div><div class="line">    &#125;</div><div class="line">  ]</div><div class="line">&#125;</div><div class="line">EOF</div></pre></td></tr></table></figure>
<blockquote>
<p>hosts：指定授权使用该证书的 etcd 节点 IP；每个节点IP 都要在里面 或者 每个机器申请一个对应IP的证书，此处选择配置所有IP进来</p>
</blockquote>
<ul>
<li>生成 etcd 证书和私钥：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div></pre></td><td class="code"><pre><div class="line">cfssl gencert -ca=ca.pem \</div><div class="line">  -ca-key=ca-key.pem \</div><div class="line">  -config=ca-config.json \</div><div class="line">  -profile=kubernetes etcd-csr.json | cfssljson -bare etcd</div><div class="line"></div><div class="line">mkdir -p /etc/etcd/ssl</div><div class="line">cp etcd.pem etcd-key.pem ca.pem /etc/etcd/ssl/</div></pre></td></tr></table></figure>
<p>接下来需要将文件拷贝到其它master节点：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line"># 先在其它主节点将该目录删除，再进行拷贝</div><div class="line">rm -rf /etc/etcd/ssl/*</div><div class="line">scp -r /etc/etcd/ssl/ 192.168.11.31:/etc/etcd/</div><div class="line">scp -r /etc/etcd/ssl/ 192.168.11.32:/etc/etcd/</div></pre></td></tr></table></figure>
<p>三个点一起执行</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">cd /root/install</div><div class="line">tar -xvf etcd-v3.1.10-linux-amd64.tar.gz</div><div class="line">mv etcd-v3.1.10-linux-amd64/etcd* /usr/local/bin</div></pre></td></tr></table></figure>
<p>创建 etcd 的 systemd unit 文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div></pre></td><td class="code"><pre><div class="line">mkdir -p /var/lib/etcd</div><div class="line">cd /var/lib/etcd</div><div class="line"></div><div class="line">cat &gt; etcd.service &lt;&lt;EOF</div><div class="line">[Unit]</div><div class="line">Description=Etcd Server</div><div class="line">After=network.target</div><div class="line">After=network-online.target</div><div class="line">Wants=network-online.target</div><div class="line">Documentation=https://github.com/coreos</div><div class="line"></div><div class="line">[Service]</div><div class="line">Type=notify</div><div class="line">WorkingDirectory=/var/lib/etcd/</div><div class="line">ExecStart=/usr/local/bin/etcd \\</div><div class="line">  --name=$&#123;NODE_NAME&#125; \\</div><div class="line">  --cert-file=/etc/etcd/ssl/etcd.pem \\</div><div class="line">  --key-file=/etc/etcd/ssl/etcd-key.pem \\</div><div class="line">  --peer-cert-file=/etc/etcd/ssl/etcd.pem \\</div><div class="line">  --peer-key-file=/etc/etcd/ssl/etcd-key.pem \\</div><div class="line">  --trusted-ca-file=/etc/etcd/ssl/ca.pem \\</div><div class="line">  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \\</div><div class="line">  --initial-advertise-peer-urls=https://$&#123;NODE_IP&#125;:2380 \\</div><div class="line">  --listen-peer-urls=https://$&#123;NODE_IP&#125;:2380 \\</div><div class="line">  --listen-client-urls=https://$&#123;NODE_IP&#125;:2379,http://127.0.0.1:2379 \\</div><div class="line">  --advertise-client-urls=https://$&#123;NODE_IP&#125;:2379 \\</div><div class="line">  --initial-cluster-token=etcd-cluster-0 \\</div><div class="line">  --initial-cluster=$&#123;ETCD_NODES&#125; \\</div><div class="line">  --initial-cluster-state=new \\</div><div class="line">  --data-dir=/var/lib/etcd</div><div class="line">Restart=on-failure</div><div class="line">RestartSec=5</div><div class="line">LimitNOFILE=65536</div><div class="line"></div><div class="line">[Install]</div><div class="line">WantedBy=multi-user.target</div><div class="line">EOF</div></pre></td></tr></table></figure>
<blockquote>
<ul>
<li>指定 etcd 的工作目录和数据目录为 /var/lib/etcd，需在启动服务前创建这个目录；</li>
<li>为了保证通信安全，需要指定 etcd 的公私钥(cert-file和key-file)、Peers 通信的公私钥和 CA 证书&gt; (peer-cert-file、peer-key-file、peer-trusted-ca-file)、客户端的CA证书（trusted-ca-file）；</li>
<li>—initial-cluster-state 值为 new 时，—name 的参数值必须位于 —initial-cluster 列表中；</li>
</ul>
</blockquote>
<ul>
<li>启动 etcd 服务</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">mv etcd.service /etc/systemd/system/</div><div class="line">systemctl daemon-reload &amp;&amp; systemctl enable etcd &amp;&amp; systemctl start etcd &amp;&amp; systemctl status etcd</div><div class="line"></div><div class="line"># 若出错</div><div class="line">systemctl status etcd.service</div><div class="line">journalctl -xe</div></pre></td></tr></table></figure>
<ul>
<li>验证服务</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">etcdctl \</div><div class="line"> --endpoints=https://$&#123;NODE_IP&#125;:2379  \</div><div class="line"> --ca-file=/etc/etcd/ssl/ca.pem \</div><div class="line"> --cert-file=/etc/etcd/ssl/etcd.pem \</div><div class="line"> --key-file=/etc/etcd/ssl/etcd-key.pem \</div><div class="line"> cluster-health</div></pre></td></tr></table></figure>
<h2 id="安装-Docker"><a href="#安装-Docker" class="headerlink" title="安装 Docker"></a>安装 Docker</h2><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div></pre></td><td class="code"><pre><div class="line"># 上传docker目录，先清除之前有的docker</div><div class="line">cd /root/install</div><div class="line">yum -y remove docker docker-common</div><div class="line"></div><div class="line">cd docker</div><div class="line">yum -y localinstall docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch.rpm</div><div class="line">yum -y localinstall docker-ce-17.03.2.ce-1.el7.centos.x86_64.rpm</div><div class="line"></div><div class="line"># 启动docker</div><div class="line">systemctl start docker &amp;&amp; systemctl enable docker</div><div class="line"># 检查状态</div><div class="line">docker ps</div><div class="line"></div><div class="line"># 配置镜像库、私库，编辑 /etc/docker/daemon.json 添加下面几项（推荐这种方式修改 docker 配置）</div><div class="line">&#123;</div><div class="line">&quot;max-concurrent-downloads&quot;: 3,</div><div class="line">&quot;max-concurrent-uploads&quot;: 5,</div><div class="line">&quot;registry-mirrors&quot;: [&quot;https://7bezldxe.mirror.aliyuncs.com/&quot;],</div><div class="line">&quot;insecure-registries&quot;: [&quot;192.168.11.200&quot;,&quot;192.168.11.218&quot;,&quot;192.168.11.188&quot;,&quot;192.168.11.230&quot;,&quot;192.168.11.112&quot;]</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<h2 id="安装-kubelet、kubectl、kubeadm、kubecni"><a href="#安装-kubelet、kubectl、kubeadm、kubecni" class="headerlink" title="安装 kubelet、kubectl、kubeadm、kubecni"></a>安装 kubelet、kubectl、kubeadm、kubecni</h2><p>切换到k8s目录，主从节点同时执行，从节点需要安装 kubeadm 从而后期可用 <code>kubeadm join</code> 加入k8s集群</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div></pre></td><td class="code"><pre><div class="line">cd ../k8s</div><div class="line">yum -y install *.rpm</div><div class="line">vi /etc/systemd/system/kubelet.service.d/10-kubeadm.conf</div><div class="line">#Environment=&quot;KUBELET_CGROUP_ARGS=--cgroup-driver=systemd&quot;</div><div class="line">Environment=&quot;KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs&quot;</div><div class="line">systemctl enable docker &amp;&amp; systemctl restart docker</div><div class="line">systemctl daemon-reload &amp;&amp; systemctl restart kubelet</div><div class="line"></div><div class="line"># 切换到docker_images目录</div><div class="line">cd ../docker_images</div><div class="line">for i in `ls`;do docker load -i $i;done</div></pre></td></tr></table></figure>
<p>在所有master节点创建 <code>config.yaml</code> 文件</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div></pre></td><td class="code"><pre><div class="line">cat &lt;&lt;EOF &gt; config.yaml</div><div class="line">apiVersion: kubeadm.k8s.io/v1alpha1</div><div class="line">kind: MasterConfiguration</div><div class="line">etcd:</div><div class="line">  endpoints:</div><div class="line">  - https://192.168.11.31:2379</div><div class="line">  - https://192.168.11.32:2379</div><div class="line">  - https://192.168.11.33:2379</div><div class="line">  caFile: /etc/etcd/ssl/ca.pem</div><div class="line">  certFile: /etc/etcd/ssl/etcd.pem</div><div class="line">  keyFile: /etc/etcd/ssl/etcd-key.pem</div><div class="line">  dataDir: /var/lib/etcd</div><div class="line">networking:</div><div class="line">  podSubnet: 10.244.0.0/16</div><div class="line">kubernetesVersion: 1.9.0</div><div class="line">api:</div><div class="line">  advertiseAddress: &quot;192.168.11.222&quot;</div><div class="line">token: &quot;b99a00.a144ef80536d4344&quot;</div><div class="line">tokenTTL: &quot;0s&quot;</div><div class="line">apiServerCertSANs:</div><div class="line">- &quot;k8s1&quot;</div><div class="line">- &quot;k8s2&quot;</div><div class="line">- &quot;k8s3&quot;</div><div class="line">- 192.168.11.31</div><div class="line">- 192.168.11.32</div><div class="line">- 192.168.11.33</div><div class="line">- 192.168.11.222</div><div class="line">featureGates:</div><div class="line">  CoreDNS: true</div><div class="line">EOF</div></pre></td></tr></table></figure>
<p>在VIP所在master节点执行以下命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">kubeadm init --config config.yaml</div><div class="line"># 记录生成的join命令</div><div class="line">kubeadm join --token b99a00.a144ef80536d4344 192.168.11.222:6443 --discovery-token-ca-cert-hash sha256:cf68966ae386e10c0233e008f21597d1d54a60ea9202d0c360a4b19fa8443328</div><div class="line"></div><div class="line">kubectl apply -f kubeadm-kuberouter.yaml</div><div class="line">kubectl get pod --all-namespaces</div></pre></td></tr></table></figure>
<p>在其它主节点执行</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">systemctl enable kubelet &amp;&amp; systemctl start kubelet</div></pre></td></tr></table></figure>
<p>启动之后，拷贝VIP所在master节点上的配置到其他master</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div></pre></td><td class="code"><pre><div class="line"># 先建立 mkdir /etc/kubernetes/pki/</div><div class="line">scp /etc/kubernetes/pki/* 192.168.11.31:/etc/kubernetes/pki/</div><div class="line">scp /etc/kubernetes/pki/* 192.168.11.32:/etc/kubernetes/pki/</div><div class="line"></div><div class="line"># 使用命令 systemctl status kubelet 确认状态保持activating (auto-restart)，即使当前没运行，但init后就会运行</div><div class="line">kubeadm init --config config.yaml # 此时三个点应该都启动了</div><div class="line"></div><div class="line"># 允许master部署pod（可选）</div><div class="line">kubectl taint nodes --all node-role.kubernetes.io/master-</div><div class="line"># 禁止master部署pod（可选）</div><div class="line">kubectl taint nodes centos-master-1 node-role.kubernetes.io/master=true:NoSchedule</div><div class="line"></div><div class="line"># 验证服务状态</div><div class="line">kubectl get nodes</div><div class="line">kubectl get cs</div><div class="line">kubectl get pods --all-namespaces</div></pre></td></tr></table></figure>
<p>在node节点执行：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">kubeadm join --token b99a00.a144ef80536d4344 192.168.11.222:6443 --discovery-token-ca-cert-hash sha256:cf68966ae386e10c0233e008f21597d1d54a60ea9202d0c360a4b19fa8443328</div></pre></td></tr></table></figure>
<h3 id="集群添加用户名密码"><a href="#集群添加用户名密码" class="headerlink" title="集群添加用户名密码"></a>集群添加用户名密码</h3><p>默认验证方式有kubeconfig和token，但这里使用basicauth的方式进行apiserver的验证</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line"># 编辑 /etc/kubernetes/pki/basic_auth_file 加入以下内容存放用户名和密码</div><div class="line">#user,password,userid</div><div class="line">admin,admin,2</div></pre></td></tr></table></figure>
<p>编辑 /etc/kubernetes/manifests/kube-apiserver.yaml，给kube-apiserver添加basic_auth验证</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">- --basic_auth_file=/etc/kubernetes/pki/basic_auth_file</div></pre></td></tr></table></figure>
<p>重启 kubelet，不重启的话，会报<code>The connection to the server 192.168.11.223:6443 was refused - did you specify the right host or port?</code></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">systemctl daemon-reload &amp;&amp; systemctl restart kubelet.service</div></pre></td></tr></table></figure>
<ul>
<li>给admin授权</li>
</ul>
<p>默认cluster-admin是拥有全部权限的，将admin和cluster-admin bind这样admin就有cluster-admin的权限。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">kubectl get clusterrole/cluster-admin -o yaml</div><div class="line">kubectl create clusterrolebinding login-on-dashboard-with-cluster-admin --clusterrole=cluster-admin --user=admin</div><div class="line">kubectl get clusterrolebinding/login-on-dashboard-with-cluster-admin -o yaml</div></pre></td></tr></table></figure>
<h3 id="k8s-注意点："><a href="#k8s-注意点：" class="headerlink" title="k8s 注意点："></a>k8s 注意点：</h3><ol>
<li><p>The connection to the server 192.168.11.160:6443 was refused - did you specify the right host or port?</p>
 <figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">#看是不是 Active: inactive (dead)</div><div class="line">systemctl status kubelet</div><div class="line">systemctl daemon-reload &amp;&amp; systemctl restart kubelet</div></pre></td></tr></table></figure>
</li>
<li><p>查看日志的命令</p>
 <figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">kubectl logs kube-apiserver --namespace=kube-system</div></pre></td></tr></table></figure>
</li>
<li><p>常用命令</p>
 <figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div></pre></td><td class="code"><pre><div class="line">kubectl get componentstatuses //查看node节点组件状态</div><div class="line">kubectl get svc -n kube-system //查看应用</div><div class="line">kubectl cluster-info //查看集群信息</div><div class="line">kubectl describe --namespace kube-system service kubernetes-dashboard //详细服务信息</div><div class="line">kubectl apply -f kube-apiserver.yaml   //更新kube-apiserver容器</div><div class="line">kubectl delete -f /root/k8s/k8s_images/kubernetes-dashboard.yaml //删除应用</div><div class="line">kubectl  delete service example-server //删除服务</div><div class="line">systemctl  start kube-apiserver.service //启动服务。</div><div class="line">kubectl get deployment --all-namespaces //启动的应用</div><div class="line">kubectl get pod  -o wide  --all-namespaces //查看pod上跑哪些服务</div><div class="line">kubectl get pod -o wide -n kube-system //查看应用在哪个node上</div><div class="line">kubectl describe pod --namespace=kube-system //查看pod上活动信息</div><div class="line">kubectl describe depoly kubernetes-dashboard -n kube-system</div><div class="line">kubectl get depoly kubernetes-dashboard -n kube-system -o yaml</div><div class="line">kubectl get service kubernetes-dashboard -n kube-system //查看应用</div><div class="line">kubectl delete -f kubernetes-dashboard.yaml //删除应用</div><div class="line">kubectl get events //查看事件</div><div class="line">kubectl get rc/kubectl get svc</div><div class="line">kubectl get namespace //获取namespace信息</div><div class="line">find -type f -print -exec grep hello &#123;&#125; \;</div><div class="line"></div><div class="line">kubeadm reset</div><div class="line">netstat -lutpn | grep 6443</div></pre></td></tr></table></figure>
</li>
</ol>
<h2 id="Harbor-安装和配置"><a href="#Harbor-安装和配置" class="headerlink" title="Harbor 安装和配置"></a>Harbor 安装和配置</h2><p>安装Harbor需要docker-compose，docker-compose可通过 pip 安装</p>
<h3 id="安装-pip-和-docker-compose"><a href="#安装-pip-和-docker-compose" class="headerlink" title="安装 pip 和 docker-compose"></a>安装 pip 和 docker-compose</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line"># 通过pip安装，先安装pip</div><div class="line">curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py</div><div class="line">python get-pip.py</div><div class="line"></div><div class="line"># 安装docker-compose</div><div class="line">pip install -U docker-compose</div></pre></td></tr></table></figure>
<blockquote>
<p>centos7 下 <code>pip install docker-compose</code> 安装 docker-compose 下报错：<br>Cannot uninstall ‘requests’. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.<br>解决办法：pip install docker-compose —ignore-installed requests</p>
</blockquote>
<h3 id="安装-harbor"><a href="#安装-harbor" class="headerlink" title="安装 harbor"></a>安装 harbor</h3><p>下载 <a href="https://storage.googleapis.com/harbor-releases/release-1.6.0/harbor-offline-installer-v1.6.0.tgz" target="_blank" rel="noopener">Harbor v1.6.0</a>，修改 harbor.cfg 使 <code>hostname = 192.168.11.200</code>，在harbor目录执行 <code>./install.sh</code> 完成安装：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line"># 查看 harbor 组件状态</div><div class="line">docker-compose ps</div><div class="line"># 启动/停止/重启harbor</div><div class="line">docker-compose start/stop/restart</div></pre></td></tr></table></figure>
<h2 id="K8S-应用日志收集——EFK-Elasticsearch-Filebeat-Kibana"><a href="#K8S-应用日志收集——EFK-Elasticsearch-Filebeat-Kibana" class="headerlink" title="K8S 应用日志收集——EFK(Elasticsearch/Filebeat/Kibana)"></a>K8S 应用日志收集——EFK(Elasticsearch/Filebeat/Kibana)</h2><h3 id="系统环境"><a href="#系统环境" class="headerlink" title="系统环境"></a>系统环境</h3><ul>
<li>3台虚拟机，一个master两个node</li>
<li>master<ul>
<li>192.168.11.218 k8s-master1-test</li>
</ul>
</li>
<li>node<ul>
<li>192.168.11.219 k8s-node1-test</li>
<li>192.168.11.221 k8s-node2-test</li>
</ul>
</li>
<li>其它<ul>
<li>3台机器全部安装jdk1.8，因为elasticsearch是java开发的</li>
<li>3台全部安装elasticsearch</li>
<li>192.168.11.218 作为主节点</li>
<li>192.168.11.219以及192.168.11.221作为数据节点</li>
<li>主节点192.168.11.218上需要安装kibana</li>
</ul>
</li>
<li>ELK版本信息：<ul>
<li>Elasticsearch-6.0.0</li>
<li>kibana-6.0.0</li>
<li>filebeat-5.4.0</li>
</ul>
</li>
</ul>
<h3 id="安装es"><a href="#安装es" class="headerlink" title="安装es"></a>安装es</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line">wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0.rpm</div><div class="line">rpm -ivh elasticsearch-6.0.0.rpm</div><div class="line"></div><div class="line">### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd</div><div class="line">sudo systemctl daemon-reload</div><div class="line">sudo systemctl enable elasticsearch.service</div><div class="line">### You can start elasticsearch service by executing</div><div class="line">sudo systemctl start elasticsearch.service</div></pre></td></tr></table></figure>
<h3 id="配置es"><a href="#配置es" class="headerlink" title="配置es"></a>配置es</h3><p>elasticsearch配置文件在这两个地方，有两个配置文件：</p>
<ul>
<li>/etc/elasticsearch/elasticsearch.yml</li>
<li>/etc/sysconfig/elasticsearch</li>
</ul>
<p>elasticsearch.yml 文件用于配置集群节点等相关信息的，elasticsearch 文件则是配置服务本身相关的配置，例如某个配置文件的路径以及java的一些路径配置</p>
<p>在 192.168.77.128 上编辑配置文件：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div></pre></td><td class="code"><pre><div class="line">cluster.name: es-master-node  # 集群中的名称</div><div class="line">node.name: k8s-master1-test  # 该节点名称</div><div class="line">node.master: true  # 意思是该节点为主节点</div><div class="line">node.data: false  # 表示这不是数据节点</div><div class="line">network.host: 0.0.0.0  # 监听全部ip，在实际环境中应设置为一个安全的ip</div><div class="line">http.port: 9200  # es服务的端口号</div><div class="line">discovery.zen.ping.unicast.hosts: [&quot;192.168.11.218&quot;, &quot;192.168.11.219&quot;, &quot;192.168.11.221&quot;] # 配置自动发现</div></pre></td></tr></table></figure>
<p>其余两节点同样编辑配置文件：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div></pre></td><td class="code"><pre><div class="line">cluster.name: es-master-node  # 集群中的名称</div><div class="line">node.name: k8s-node1-test  # 该节点名称</div><div class="line">node.master: false  # 意思是该节点为主节点</div><div class="line">node.data: true  # 表示这不是数据节点</div><div class="line">network.host: 0.0.0.0  # 监听全部ip，在实际环境中应设置为一个安全的ip</div><div class="line">http.port: 9200  # es服务的端口号</div><div class="line">discovery.zen.ping.unicast.hosts: [&quot;192.168.11.218&quot;, &quot;192.168.11.219&quot;, &quot;192.168.11.221&quot;] # 配置自动发现</div></pre></td></tr></table></figure>
<p>安装完启动：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">systemctl start elasticsearch.service</div></pre></td></tr></table></figure>
<p>curl查看es集群情况:</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div></pre></td><td class="code"><pre><div class="line">curl &apos;localhost:9200/_cluster/health?pretty&apos;</div><div class="line"># 返回结果</div><div class="line">&#123;</div><div class="line">  &quot;cluster_name&quot; : &quot;es-master-node&quot;,</div><div class="line">  &quot;status&quot; : &quot;yellow&quot;,</div><div class="line">  &quot;timed_out&quot; : false,</div><div class="line">  &quot;number_of_nodes&quot; : 2,</div><div class="line">  &quot;number_of_data_nodes&quot; : 1,</div><div class="line">  &quot;active_primary_shards&quot; : 1,</div><div class="line">  &quot;active_shards&quot; : 1,</div><div class="line">  &quot;relocating_shards&quot; : 0,</div><div class="line">  &quot;initializing_shards&quot; : 0,</div><div class="line">  &quot;unassigned_shards&quot; : 1,</div><div class="line">  &quot;delayed_unassigned_shards&quot; : 0,</div><div class="line">  &quot;number_of_pending_tasks&quot; : 0,</div><div class="line">  &quot;number_of_in_flight_fetch&quot; : 0,</div><div class="line">  &quot;task_max_waiting_in_queue_millis&quot; : 0,</div><div class="line">  &quot;active_shards_percent_as_number&quot; : 50.0</div><div class="line">&#125;</div></pre></td></tr></table></figure>
<p>查看集群的详细信息：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">curl &apos;192.168.11.218:9200/_cluster/state?pretty&apos;</div></pre></td></tr></table></figure>
<h3 id="安装-kibana"><a href="#安装-kibana" class="headerlink" title="安装 kibana"></a>安装 kibana</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">wget https://artifacts.elastic.co/downloads/kibana/kibana-6.0.0-x86_64.rpm</div><div class="line">rpm -ivh kibana-6.0.0-x86_64.rpm</div></pre></td></tr></table></figure>
<h3 id="配置-kibana"><a href="#配置-kibana" class="headerlink" title="配置 kibana"></a>配置 kibana</h3><p>编辑 <code>vim /etc/kibana/kibana.yml</code> 增加以下内容</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">server.port: 5601  # 配置kibana的端口</div><div class="line">server.host: 192.168.11.218  # 配置监听ip</div><div class="line">elasticsearch.url: &quot;http://192.168.11.218:9200&quot;  # 配置es服务器的ip，如果是集群则配置该集群中主节点的ip</div><div class="line">logging.dest: /var/log/kibana.log  # 配置kibana的日志文件路径，不然默认是messages里记录日志</div></pre></td></tr></table></figure>
<p>启动：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">systemctl start kibana</div></pre></td></tr></table></figure>
<p>浏览器里进行访问 <code>http://192.168.11.218:5601/</code> 即可</p>
<ul>
<li>常用：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">#查看es索引：</div><div class="line">curl &apos;192.168.11.218:9200/_cat/indices?v&apos;</div><div class="line">#获取指定索引详细信息：</div><div class="line">curl -XGET &apos;192.168.77.128:9200/system-syslog-2018.03?pretty&apos;</div><div class="line">#需要删除索引的话，使用以下命令可以删除指定索引：</div><div class="line">curl -XDELETE &apos;localhost:9200/logcollection-test&apos;</div></pre></td></tr></table></figure>
<h2 id="日志收集"><a href="#日志收集" class="headerlink" title="日志收集"></a>日志收集</h2><h3 id="制作-filebeat-镜像："><a href="#制作-filebeat-镜像：" class="headerlink" title="制作 filebeat 镜像："></a>制作 filebeat 镜像：</h3><p>首先准备 filebeat-5.4.0-linux-x86_64.tar.gz</p>
<ul>
<li>docker-entrypoint.sh</li>
</ul>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div></pre></td><td class="code"><pre><div class="line"><span class="meta">#!/bin/bash</span></div><div class="line">config=/etc/filebeat/filebeat.yml</div><div class="line">env</div><div class="line"><span class="built_in">echo</span> <span class="string">'Filebeat init process done. Ready for start up.'</span></div><div class="line"><span class="built_in">echo</span> <span class="string">"Using the following configuration:"</span></div><div class="line">cat /etc/filebeat/filebeat.yml</div><div class="line"><span class="built_in">exec</span> <span class="string">"<span class="variable">$@</span>"</span></div></pre></td></tr></table></figure>
<ul>
<li>dockerfile</li>
</ul>
<figure class="highlight dockerfile"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">FROM</span> centos</div><div class="line"><span class="keyword">MAINTAINER</span> YangLiangWei &lt;ylw@fjhb.cn&gt;</div><div class="line"></div><div class="line"><span class="comment"># Install Filebeat</span></div><div class="line"><span class="keyword">WORKDIR</span><span class="bash"> /usr/<span class="built_in">local</span></span></div><div class="line"><span class="keyword">COPY</span><span class="bash"> filebeat-5.4.0-linux-x86_64.tar.gz  /usr/<span class="built_in">local</span></span></div><div class="line"><span class="keyword">RUN</span><span class="bash"> <span class="built_in">cd</span> /usr/<span class="built_in">local</span> &amp;&amp; \</span></div><div class="line">    tar xvf filebeat-5.4.0-linux-x86_64.tar.gz &amp;&amp; \</div><div class="line">    rm <span class="_">-f</span> filebeat-5.4.0-linux-x86_64.tar.gz &amp;&amp; \</div><div class="line">    ln <span class="_">-s</span> /usr/<span class="built_in">local</span>/filebeat-5.4.0-linux-x86_64 /usr/<span class="built_in">local</span>/filebeat &amp;&amp; \</div><div class="line">    chmod +x /usr/<span class="built_in">local</span>/filebeat/filebeat &amp;&amp; \</div><div class="line">    mkdir -p /etc/filebeat</div><div class="line"></div><div class="line"><span class="keyword">ADD</span><span class="bash"> ./docker-entrypoint.sh /usr/bin/</span></div><div class="line"><span class="keyword">RUN</span><span class="bash"> chmod +x /usr/bin/docker-entrypoint.sh</span></div><div class="line"></div><div class="line"><span class="keyword">ENTRYPOINT</span><span class="bash"> [<span class="string">"docker-entrypoint.sh"</span>]</span></div><div class="line"></div><div class="line"><span class="keyword">CMD</span><span class="bash"> [<span class="string">"/usr/local/filebeat/filebeat"</span>,<span class="string">"-e"</span>,<span class="string">"-c"</span>,<span class="string">"/etc/filebeat/filebeat.yml"</span>]</span></div></pre></td></tr></table></figure>
<ul>
<li>build docker 镜像</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">docker build -t filebeat:v5.4.0</div><div class="line">docker tag filebeat:v5.4.0 192.168.11.218/hlg_web/filebeat:v5.4.0</div></pre></td></tr></table></figure>
<h3 id="配置-k8s-Secret-和私库认证"><a href="#配置-k8s-Secret-和私库认证" class="headerlink" title="配置 k8s Secret 和私库认证"></a>配置 k8s Secret 和私库认证</h3><figure class="highlight yaml"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line"><span class="comment"># 其中 .dockerconfigjson 字段值可以在login到私库后用 cat /root/.docker/config.json | base64 -w 0 获取</span></div><div class="line"><span class="attr">kind:</span> Secret</div><div class="line"><span class="attr">apiVersion:</span> v1</div><div class="line"><span class="attr">metadata:</span></div><div class="line"><span class="attr">  name:</span> regsecret</div><div class="line"><span class="attr">type:</span> kubernetes.io/dockerconfigjson</div><div class="line"><span class="attr">data:</span></div><div class="line">  <span class="string">".dockerconfigjson"</span>: ewoJImF1dGhzIjogewoJCSIxOTIuMTY4LjExLjIxOCI6IHsKCQkJImF1dGgiOiAiWVdSdGFXNDZTR0Z5WW05eU1USXpORFU9IgoJCX0KCX0KfQ==</div></pre></td></tr></table></figure>
<p>这样，以后若在pod中定义的镜像需要拉取私库私有镜像则只要添加<code>imagePullSecrets</code>字段即可：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div></pre></td><td class="code"><pre><div class="line"><span class="attr">containers:</span></div><div class="line"><span class="attr">- image:</span> <span class="number">192.168</span><span class="number">.11</span><span class="number">.200</span>/hlg_web/datalower:<span class="number">1.1</span></div><div class="line"><span class="attr">  name:</span> app</div><div class="line"><span class="attr">  ports:</span></div><div class="line"><span class="attr">  - containerPort:</span> <span class="number">80</span></div><div class="line"><span class="attr">  volumeMounts:</span></div><div class="line"><span class="attr">  - name:</span> app-logs</div><div class="line"><span class="attr">    mountPath:</span> /app/log</div><div class="line"><span class="attr">- image:</span> <span class="number">192.168</span><span class="number">.11</span><span class="number">.200</span>/hlg_web/filebeat:v5<span class="number">.4</span><span class="number">.0</span></div><div class="line"><span class="attr">  name:</span> filebeat</div><div class="line"><span class="attr">  volumeMounts:</span></div><div class="line"><span class="attr">  - name:</span> app-logs</div><div class="line"><span class="attr">    mountPath:</span> /log</div><div class="line"><span class="attr">  - name:</span> filebeat-config</div><div class="line"><span class="attr">    mountPath:</span> /etc/filebeat/</div><div class="line"><span class="attr">volumes:</span></div><div class="line"><span class="attr">- name:</span> app-logs</div><div class="line"><span class="attr">  emptyDir:</span> &#123;&#125;</div><div class="line"><span class="attr">- name:</span> filebeat-config</div><div class="line"><span class="attr">  configMap:</span></div><div class="line"><span class="attr">    name:</span> filebeat-config</div><div class="line"><span class="attr">imagePullSecrets:</span></div><div class="line"><span class="attr">- name:</span> regsecret</div></pre></td></tr></table></figure>
<h3 id="Deployment-配置"><a href="#Deployment-配置" class="headerlink" title="Deployment 配置"></a>Deployment 配置</h3><figure class="highlight yaml"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div><div class="line">42</div><div class="line">43</div><div class="line">44</div><div class="line">45</div><div class="line">46</div><div class="line">47</div></pre></td><td class="code"><pre><div class="line"><span class="attr">apiVersion:</span> extensions/v1beta1</div><div class="line"><span class="attr">kind:</span> Deployment</div><div class="line"><span class="attr">metadata:</span></div><div class="line"><span class="attr">  name:</span> logcollection-test</div><div class="line"><span class="attr">  namespace:</span> default</div><div class="line"><span class="attr">spec:</span></div><div class="line"><span class="attr">  replicas:</span> <span class="number">1</span></div><div class="line"><span class="attr">  template:</span></div><div class="line"><span class="attr">    metadata:</span></div><div class="line"><span class="attr">      labels:</span></div><div class="line"><span class="attr">        run:</span> logcollection-test</div><div class="line"><span class="attr">    spec:</span></div><div class="line"><span class="attr">      containers:</span></div><div class="line"><span class="attr">      - image:</span> <span class="number">192.168</span><span class="number">.11</span><span class="number">.200</span>/hlg_web/datalower:<span class="number">1.1</span></div><div class="line"><span class="attr">        name:</span> app</div><div class="line"><span class="attr">        ports:</span></div><div class="line"><span class="attr">        - containerPort:</span> <span class="number">80</span></div><div class="line"><span class="attr">        volumeMounts:</span></div><div class="line"><span class="attr">        - name:</span> app-logs</div><div class="line"><span class="attr">          mountPath:</span> /app/log</div><div class="line"><span class="attr">      - image:</span> <span class="number">192.168</span><span class="number">.11</span><span class="number">.200</span>/hlg_web/filebeat:v5<span class="number">.4</span><span class="number">.0</span></div><div class="line"><span class="attr">        name:</span> filebeat</div><div class="line"><span class="attr">        volumeMounts:</span></div><div class="line"><span class="attr">        - name:</span> app-logs</div><div class="line"><span class="attr">          mountPath:</span> /log</div><div class="line"><span class="attr">        - name:</span> filebeat-config</div><div class="line"><span class="attr">          mountPath:</span> /etc/filebeat/</div><div class="line"><span class="attr">      volumes:</span></div><div class="line"><span class="attr">      - name:</span> app-logs</div><div class="line"><span class="attr">        emptyDir:</span> &#123;&#125;</div><div class="line"><span class="attr">      - name:</span> filebeat-config</div><div class="line"><span class="attr">        configMap:</span></div><div class="line"><span class="attr">          name:</span> filebeat-config</div><div class="line"><span class="meta">---</span></div><div class="line"><span class="attr">apiVersion:</span> v1</div><div class="line"><span class="attr">kind:</span> ConfigMap</div><div class="line"><span class="attr">metadata:</span></div><div class="line"><span class="attr">  name:</span> filebeat-config</div><div class="line"><span class="attr">data:</span></div><div class="line">  filebeat.yml: <span class="string">|</span></div><div class="line">    filebeat.prospectors:</div><div class="line"><span class="attr">    - input_type:</span> log</div><div class="line"><span class="attr">      paths:</span></div><div class="line"><span class="bullet">        -</span> <span class="string">"/log/*"</span></div><div class="line">    output.elasticsearch:</div><div class="line"><span class="attr">      hosts:</span> [<span class="string">"192.168.11.218:9200"</span>]</div><div class="line"><span class="attr">      index:</span> <span class="string">"logcollection-test"</span></div></pre></td></tr></table></figure>
<h2 id="参考："><a href="#参考：" class="headerlink" title="参考："></a>参考：</h2><ul>
<li><a href="https://blog.csdn.net/tiger435/article/details/78517135" target="_blank" rel="noopener">kubeadm快速部署kubernetes(HA)</a></li>
<li><a href="http://blog.51cto.com/net592/2063748" target="_blank" rel="noopener">[K8s 1.9实践]Kubeadm 1.9 HA 高可用 集群 本地离线镜像部署</a></li>
<li><a href="https://blog.csdn.net/ywq935/article/details/79856469" target="_blank" rel="noopener">k8s（一）、 1.9.0高可用集群本地离线部署记录</a></li>
<li><a href="https://www.cnblogs.com/puroc/p/5431375.html" target="_blank" rel="noopener">Harbor的搭建（vmware企业级docker镜像私服）</a></li>
<li><a href="http://blog.51cto.com/zero01/2079879" target="_blank" rel="noopener">搭建ELK日志分析平台（上）—— ELK介绍及搭建 Elasticsearch 分布式集群</a></li>
<li><a href="http://blog.51cto.com/zero01/2082794" target="_blank" rel="noopener">搭建ELK日志分析平台（下）—— 搭建kibana和logstash服务器</a></li>
<li><a href="http://blog.51cto.com/ylw6006/2107307" target="_blank" rel="noopener">K8S使用filebeat统一收集应用日志</a></li>
<li><a href="https://jimmysong.io/kubernetes-handbook/practice/app-log-collection.html" target="_blank" rel="noopener">应用日志收集· Kubernetes Handbook - Kubernetes中文指南</a></li>
</ul>

      
    </div>
    
    
    

    

    
      <div>
        <div style="padding: 10px 0; margin: 20px auto; width: 90%; text-align: center;">
  <div>请我喝杯咖啡吧☕~</div>
  <button id="rewardButton" disable="enable" onclick="var qr = document.getElementById('QR'); if (qr.style.display === 'none') {qr.style.display='block';} else {qr.style.display='none'}">
    <span>打赏</span>
  </button>
  <div id="QR" style="display: none;">

    
      <div id="wechat" style="display: inline-block">
        <img id="wechat_qr" src="/images/wechatpay.jpg" alt="Fighter. 微信支付"/>
        <p>微信支付</p>
      </div>
    

    
      <div id="alipay" style="display: inline-block">
        <img id="alipay_qr" src="/images/alipay.jpg" alt="Fighter. 支付宝"/>
        <p>支付宝</p>
      </div>
    

    

  </div>
</div>

      </div>
    

    
      <div>
        <ul class="post-copyright">
  <li class="post-copyright-author">
    <strong>本文作者：</strong>
    Fighter.
  </li>
  <li class="post-copyright-link">
    <strong>本文链接：</strong>
    <a href="http://fighterhit.github.io/2018/12/04/other_notes/Kubernetes-记录/" title="Kubernetes 记录">http://fighterhit.github.io/2018/12/04/other_notes/Kubernetes-记录/</a>
  </li>
  <li class="post-copyright-license">
    <strong>版权声明： </strong>
    本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/3.0/" rel="external nofollow" target="_blank">CC BY-NC-SA 3.0</a> 许可协议。转载请注明出处！
  </li>
</ul>

      </div>
    

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/Kubernetes/" rel="tag"># Kubernetes</a>
          
            <a href="/tags/Docker/" rel="tag"># Docker</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2018/06/20/other_notes/制作Docker镜像/" rel="next" title="制作Docker镜像">
                <i class="fa fa-chevron-left"></i> 制作Docker镜像
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
        <!-- JiaThis Button BEGIN -->
<div class="jiathis_style">
<span class="jiathis_txt">分享到：</span>
<a class="jiathis_button_fav">收藏夹</a>
<a class="jiathis_button_tsina">新浪微博</a>
<a class="jiathis_button_qzone">QQ空间</a>
<a class="jiathis_button_weixin">微信</a>
<a class="jiathis_button_cqq">腾讯QQ</a>
<a class="jiathis_button_copy">复制网址</a>
<a class="jiathis_button_share">一键分享</a>

<a href="http://www.jiathis.com/share?uid=2140465" class="jiathis jiathis_txt jiathis_separator jtico jtico_jiathis" target="_blank">更多</a>
<a class="jiathis_counter_style"></a>
</div>
<script type="text/javascript" >
var jiathis_config={
  data_track_clickback:true,
  summary:"",
  shortUrl:false,
  hideMore:false
}
</script>
<script type="text/javascript" src="http://v3.jiathis.com/code/jia.js?uid=" charset="utf-8"></script>
<!-- JiaThis Button END -->
      
    </div>
  </div>


          </div>
          


          
  
    <div class="comments" id="comments">
      <div id="lv-container" data-id="city" data-uid="MTAyMC8zMjY0Ni85MjA3"></div>
    </div>
  


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image"
                src="/images/avatar.jpg"
                alt="Fighter." />
            
              <p class="site-author-name" itemprop="name">Fighter.</p>
              <p class="site-description motion-element" itemprop="description">学习 | 分享 | 交流 | 进步</p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
              
                <a href="/archives/">
              
                  <span class="site-state-item-count">26</span>
                  <span class="site-state-item-name">日志</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-categories">
                <a href="/categories/index.html">
                  <span class="site-state-item-count">6</span>
                  <span class="site-state-item-name">分类</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-tags">
                <a href="/tags/index.html">
                  <span class="site-state-item-count">28</span>
                  <span class="site-state-item-name">标签</span>
                </a>
              </div>
            

          </nav>

          

          <div class="links-of-author motion-element">
            
              
                <span class="links-of-author-item">
                  <a href="https://github.com/fighterhit" target="_blank" title="GitHub">
                    
                      <i class="fa fa-fw fa-github"></i>GitHub</a>
                </span>
              
                <span class="links-of-author-item">
                  <a href="https://weibo.com/fighterhit" target="_blank" title="Weibo">
                    
                      <i class="fa fa-fw fa-weibo"></i>Weibo</a>
                </span>
              
            
          </div>

          
          

          
          

          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#环境"><span class="nav-number">1.</span> <span class="nav-text">环境</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#系统配置"><span class="nav-number">1.1.</span> <span class="nav-text">系统配置</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装-keepalived（只对master节点操作）"><span class="nav-number">2.</span> <span class="nav-text">安装 keepalived（只对master节点操作）</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#安装"><span class="nav-number">2.1.</span> <span class="nav-text">安装</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#修改配置文件-etc-keepalived-keepalived-conf"><span class="nav-number">2.2.</span> <span class="nav-text">修改配置文件 /etc/keepalived/keepalived.conf</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#依次启动"><span class="nav-number">2.3.</span> <span class="nav-text">依次启动</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装Etcd-https（只对master节点操作）"><span class="nav-number">3.</span> <span class="nav-text">安装Etcd https（只对master节点操作）</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#Etcd-证书"><span class="nav-number">3.1.</span> <span class="nav-text">Etcd 证书</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装-Docker"><span class="nav-number">4.</span> <span class="nav-text">安装 Docker</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装-kubelet、kubectl、kubeadm、kubecni"><span class="nav-number">5.</span> <span class="nav-text">安装 kubelet、kubectl、kubeadm、kubecni</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#集群添加用户名密码"><span class="nav-number">5.1.</span> <span class="nav-text">集群添加用户名密码</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#k8s-注意点："><span class="nav-number">5.2.</span> <span class="nav-text">k8s 注意点：</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Harbor-安装和配置"><span class="nav-number">6.</span> <span class="nav-text">Harbor 安装和配置</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#安装-pip-和-docker-compose"><span class="nav-number">6.1.</span> <span class="nav-text">安装 pip 和 docker-compose</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装-harbor"><span class="nav-number">6.2.</span> <span class="nav-text">安装 harbor</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#K8S-应用日志收集——EFK-Elasticsearch-Filebeat-Kibana"><span class="nav-number">7.</span> <span class="nav-text">K8S 应用日志收集——EFK(Elasticsearch/Filebeat/Kibana)</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#系统环境"><span class="nav-number">7.1.</span> <span class="nav-text">系统环境</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装es"><span class="nav-number">7.2.</span> <span class="nav-text">安装es</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#配置es"><span class="nav-number">7.3.</span> <span class="nav-text">配置es</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装-kibana"><span class="nav-number">7.4.</span> <span class="nav-text">安装 kibana</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#配置-kibana"><span class="nav-number">7.5.</span> <span class="nav-text">配置 kibana</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#日志收集"><span class="nav-number">8.</span> <span class="nav-text">日志收集</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#制作-filebeat-镜像："><span class="nav-number">8.1.</span> <span class="nav-text">制作 filebeat 镜像：</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#配置-k8s-Secret-和私库认证"><span class="nav-number">8.2.</span> <span class="nav-text">配置 k8s Secret 和私库认证</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#Deployment-配置"><span class="nav-number">8.3.</span> <span class="nav-text">Deployment 配置</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#参考："><span class="nav-number">9.</span> <span class="nav-text">参考：</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      
      
    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; 2016 - <span itemprop="copyrightYear">2018</span>
  <span class="with-love">
    <i class="fa fa-home"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Fighter.</span>

  
</div>









        
<div class="busuanzi-count">
  <!--<script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js"></script>-->
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="site-uv">
      <i class="fa fa-user"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
      
    </span>
  

  
    <span class="site-pv">
      <i class="fa fa-eye"></i>
      <span class="busuanzi-value" id="busuanzi_value_site_pv"></span>
      
    </span>
  

    <span class="site-pv">&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;
    Hosted by <a target="_blank" href="https://github.com/">GitHub Pages</a>
    </span>

</div>







        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.3"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.3"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.3"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.3"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.3"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.3"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.3"></script>



  


  




	





  





  
    <script type="text/javascript">
      (function(d, s) {
        var j, e = d.getElementsByTagName(s)[0];
        if (typeof LivereTower === 'function') { return; }
        j = d.createElement(s);
        j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
        j.async = true;
        e.parentNode.insertBefore(j, e);
      })(document, 'script');
    </script>
  








  





  

  
<script>
(function(){
    var bp = document.createElement('script');
    var curProtocol = window.location.protocol.split(':')[0];
    if (curProtocol === 'https') {
        bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';        
    }
    else {
        bp.src = 'http://push.zhanzhang.baidu.com/push.js';
    }
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(bp, s);
})();
</script><!-- hexo-inject:begin --><!-- hexo-inject:end -->


  

  
  


  

  

</body>
</html>
